# This is an experimental file, I've been developing to test
# how feesible the use of ragel will be in parsing text, typically html files with embedded esi markup
#
# The benefit from using ragel:
# 
# We should be able to craft the parser in such away to avoid holding the full document in memory.
# It should be fast and language independent: (C/C++,Java,Ruby,etc...)
# 
# Being able to view the state machine as a graph can be useful for spotting potential holes.
#
# The downside to using ragel:
#
# So, far as I can tell it is not simple or easy to understand. 
# Finate state machines are delicate and must be constructed very carefully.
#
# At the end of all this we really just would like something to emit tags 
# to our tag classes defined in lib/esi/tags, to make it easier to 
#  1. test the esi capablities
#  2. implement more of the esi spec
#  3. keep the server reliable
#  4. keep the code base small
#

class ESIParser
  attr_accessor :output, :esi_tag

%%{
  machine esi;

  action begin {
    @mark = p
  }
  action finish {
  }

  # record the position of the start tag
  action see_start_tag {
    @tag_text = data[@mark,p-@mark]
    @tag_info = {} # store the tag attributes
    @tag_info[:name] = @tag_text.sub('<','').strip
    @tag_info[:attributes] = {}
#    puts "have esi tag at #{p}=>#{@mark}:#{data[p,1].inspect} with data #{@tag_text.inspect}"
    @mark = p
  }

  action see_end_tag {
#    puts "parsed esi tag at #{p}=>#{@mark}:#{data[p,1].inspect} with data #{@tag_text.inspect}"
    @start_tag.call @tag_info[:name], @tag_info[:attributes] if @start_tag
    @end_tag.call @tag_info[:name] if @end_tag
    @attr_key = nil
    @attr_value = nil
    @tag_text = nil
    @tag_info = nil
  }

  action see_close_start_tag {
    @start_tag.call @tag_info[:name], @tag_info[:attributes] if @start_tag
    @attr_key = nil
    @attr_value = nil
    @tag_text = nil
    @tag_info = nil
  }

  action see_attribute_key {
    @attr_key = data[@mark,p-@mark]
    @mark += (@attr_key.size)
    @attr_key.gsub!(/^['"]/,'')
    @attr_key.strip!
#    puts "SeeAttributeKey: #{data[@mark,1].inspect}: #{data[p,1].inspect} #{@attr_key.inspect}"
  }

  action see_attribute_value {
    @attr_value = data[@mark,p-@mark]
    @attr_value.strip!
    @attr_value.gsub!(/^=?\s*['"]/,'')
    @attr_value.gsub!(/['"]$/,'')

    @tag_info[:attributes][@attr_key] = @attr_value
#    puts "SeeAttributeValue: #{p} #{data[@mark,1].inspect}: #{data[p,1].inspect} #{@attr_key.inspect} => #{@attr_value.inspect}"
    @mark = p
  }

  action block_start_tag {
    tag_text = data[@mark+1,p-@mark-1]
    @start_tag.call tag_text, {} if @start_tag
#    puts "Block start: #{p} #{tag_text}"
  }

  action block_end_tag {
    tag_text = data[@mark+2,p-@mark-2]
    @end_tag.call tag_text if @end_tag
#    puts "Block end: #{p} #{tag_text}"
  }

  action echo {
    case cs
      when 1 then @prev_buffer = data[p,1] # detected '<'
      when 2 then @prev_buffer << data[p,1] # detected '/' following a '<'
      when 0 # the start state
        stream_buffer @prev_buffer if @prev_buffer
        # NOTE: state 12 is the character state before <esi:try>, state 7 is the character before </esi:try>
        #                                                     |                                           |
        #                                                     - state 12                                  - state 7
        stream_buffer data[p,1] if @prev_state != 12 and @prev_state != 7
        @prev_buffer = nil
        @prev_state = nil
      else
        @prev_buffer = nil
        @prev_state = cs
    end
    print "'#{data[p,1]}:#{cs}'," if $debug
  }

  # inline tags e.g. <esi:comment text='some useless text'/> or <esi:include src='/foo/'/>, etc...
  esi_inline_tag = (
    start: (
      '<esi:' [a-z]+ space @see_start_tag -> start_attribute |
      '<esi:' [a-z]+ space* '/>' >see_start_tag %see_end_tag -> final
    ),
    start_attribute: (
      space* [a-zA-Z_\-]+ [a-zA-Z_\-0-9]* '=' @see_attribute_key -> end_attribute | '/>' @see_end_tag -> final
    ),
    end_attribute: (
      space* ['"][!#-&(-/:-@a-zA-Z0-9+_\-]+['"] @see_attribute_value -> start_attribute
    )
  ) >begin %finish;

  # block tags e.g. <esi:try>  <esi:attempt>, etc..
  esi_block_tag = (
    start: (
      '<esi:' [a-z]+ '>' @block_start_tag -> start |
      '<esi:' [a-z]+ space @see_start_tag -> start_attribute |
      '</esi:' [a-z]+ '>' @block_end_tag
    ),
    start_attribute: (
      space* [a-zA-Z_\-]+ [a-zA-Z_\-0-9]* '=' @see_attribute_key -> end_attribute | '>' @see_close_start_tag -> final
    ),
    end_attribute: (
      space* ['"][!#-&(-/:-@a-zA-Z0-9+_\-]+['"] @see_attribute_value -> start_attribute
    )
  ) >begin  %finish;

  echo_state = (
    start: (
      /./ @echo
    )
  );

#  tag_end = '/>';
#  tag_start_expr = '<esi:' @start_reading_tag_name;
#  read_tag := word+ any-space+ @end_reading_tag_name ;
  main := ((/.*/)? @echo (esi_inline_tag|esi_block_tag) )*;
}%%

  attr_reader :tags

  def initialize( output_stream )
    @data = nil
    @prev_buffer = nil
    @output = output_stream
    @start_tag = nil # call back for when we encounter a start tag
    @end_tag = nil # call back for when we encounter an end tag
    @esi_tag = nil # the current active esi tag
  end

  def output=(output_stream)
    @output = output_stream
  end

  def start_tag(&block)
    @start_tag = block
  end
  
  def end_tag(&block)
    @end_tag = block
  end

  # process a block of esi tags
  def process(data)
    if @data
#      puts "append : #{@mark} : #{p}"
      data = @data + data
      p = @data.length
    end
    %% write data;
    @mark ||= 0
    p ||= 0
    pe ||= data.length
    @cs ||= esi_start
    cs = @cs
#    puts "process: #{cs.inspect} :start #{data.inspect}, #{p}"
    %% write exec;
    @cs = cs
    if (@cs != _esi_trans_actions_wi.last && @cs !=  esi_start && @cs != 0)
#    puts "append process: #{@cs.inspect}"
      @data = data
    else
      @data = nil
    end
  end

  def finish
    %% write eof;
  end

private
  def stream_buffer(string)
    if @esi_tag
      @esi_tag.buffer( @output, string )# unless string.strip.empty?
    else
      @output << string
    end
  end

end

require 'webrick'
class ::WEBrick::HTTPServer ; def access_log(config, req, res) ; end ; end
class ::WEBrick::BasicLog ; def log(level, data) ; end ; end

require 'test/unit'

module TestServer
  def setup
    @server = WEBrick::HTTPServer.new( :Port => 99999 )

    # setup test server (simulates exact target)
    @server.mount_proc("/test/error") do|req,res|
      raise "fail"
    end
    @server.mount_proc("/test/redirect") do|req,res|
      res.set_redirect(WEBrick::HTTPStatus::MovedPermanently,"/test/success")
    end
    @server.mount_proc("/test/redirect_to_error") do|req,res|
      res.set_redirect(WEBrick::HTTPStatus::MovedPermanently,"/test/error")
    end
    @server.mount_proc("/test/success") do|req,res|
      id = req.query["id"]
      if id
        res.body = "<div id='#{id}'>id string</div>"
      else
        res.body = "<div>hello there world</div>"
      end
      res['Content-Type'] = "text/html"
    end
    @server.mount_proc("/test/timeout") do|req,res|
      sleep 1.5
      res.body = "<div>We won't get this for a few seconds</div>"
      res['Content-Type'] = "text/html"
    end
    @server.mount_proc("/test/headers") do|req,res|
      res.body = "<div>hello there world</div>"
      res['Content-Type'] = "text/html"
      begin
        assert_equal ['value1'], req.header['headers1']
        assert_equal ['value2'], req.header['headers2']
      rescue Object => e
        puts e.message
        raise e
      end
    end

    # start up the server in a background thread
    @thread = Thread.new(@server) do|server|
      server.start
    end

    setup_extra if self.respond_to?(:setup_extra)
  end
  def teardown
    @server.shutdown
    @thread.join
  end
end
 

#
# Some thoughts about parser interface
# 
# Goals:
#  * keep the parser separate from any HTTP or network processing
#  * pass or collection the information that's interesting e.g tag start/end
#  * read from an input stream
#  * write non esi tag markup to an output stream
#  * make sure the output stream can be overridable so that try or attempt/except tags can control where the output is streamed
#
# Usage:
#  
#  parser = ESIParser.new output_stream
#
#  parser.start_tag = do|tag_name,attrs|
#  end
#
#  parser.end_tag = do|tag_name|
#  end
#
#  parser.output = output_stream # change the output stream from a start_tag or end_tag handler  # TODO
#
#  # stream data to the parser as it's made available
#  response.read_body do|s|
#    parser.process s
#  end
require 'yaml'
require 'test/unit'
$:.unshift File.join(File.dirname(__FILE__), "lib")
require 'esi/cache'
require 'esi/router'
require 'esi/tag/base'
require 'esi/tag/include'
require 'esi/tag/attempt'
require 'esi/tag/except'
require 'esi/tag/invalidate'
require 'esi/tag/try'

class TestInlineTagParser < Test::Unit::TestCase
  include TestServer

  def test_with_tags
    router = ESI::Router.new(YAML.load(%q(
  default:
    host: 127.0.0.1
    port: 99999) ) )
    sample =  File.join(File.dirname(__FILE__),'esi-sample.html')
    cache = ESI::RubyCache.new
    File.open('out-sample.html','w') do|output|
      parser = ESIParser.new output

      parser.start_tag do|tag_name,attrs|
        tag = ESI::Tag::Base.create(router, {}, {}, tag_name.gsub(/esi:/,''), attrs, cache)
        if parser.esi_tag
          parser.esi_tag.add_child(tag)
        else
          parser.esi_tag = tag
        end
      end

      parser.end_tag do|tag_name|
        if parser.esi_tag.name == tag_name.gsub(/esi:/,'')
          parser.esi_tag.close(parser.output)
          parser.esi_tag = nil
        else
          parser.esi_tag.close_child(parser.output,tag_name)
        end
      end
      File.open(sample,'r') do|input|
        while( (buf=input.read(128)) and !input.eof? and buf.size != 0 )
          parser.process buf
        end
        parser.process buf
      end
      parser.finish
    end
    output = File.read("out-sample.html")

    assert_match(/  <div class="body">/, output)
    assert_match(/    <div>some content<\/div>/, output)
    assert_match(/    <!-- a little commentary -->/, output)
    assert_match(%q(<!-- a 
      multiline comment -->),output)
    assert_match(%q(<![CDATA[
      Some cdata fun fun fun
    ]]>), output)
    assert_match(/<!DOCTYPE html PUBLIC "-\/\/W3C\/\/DTD XHTML 1.0 Strict\/\/EN" "http:\/\/www.w3.org\/TR\/xhtml1\/DTD\/xhtml1-strict.dtd">/, output)
    assert_no_match(/<esi:/, output)
    assert_match(/<div id='1'>id string<\/div>/, output)
    assert_match(/<div id='3'>id string<\/div>/, output)

    assert_no_match(/<p>it failed<\/p>/, output)
    assert_match(/<div id='2'>id string<\/div>/, output)

    assert_match(/<div id='4'>id string<\/div>/, output)
    assert_match(/<p>We should get this or not\?<\/p>/, output)
    assert_match(/<p>Now maybe we shouldn't see this\?<\/p>/, output)
    assert_match(/<p>except worked1<\/p>/, output)
    assert_match(/<p>except worked2<\/p>/, output)
    assert_match(/<\/html>/, output)
  end

  def test_inline_parse_basics
    output = ""
    parser = ESIParser.new output
    tags = []
    parser.start_tag do|tag_name, attrs|
      tags << {:name => tag_name, :attributes => attrs}
    end

    parser.process "<html><head><body><esi:include timeout='1' max-age='600+600' src=\"hello\"/>some more input"
    parser.process "some input<esi:include \nsrc='hello'/>some more input\nsome input<esi:include src=\"hello\"/>some more input"
    parser.process "some input<esi:inline src='hello'/>some more input\nsome input<esi:comment text='hello'/>some more input"
    parser.process "<p>some input</p><esi:include src='hello'/>some more input\nsome input<esi:include src='hello'/>some more input"
    parser.process "</body></html>"
    parser.finish
    assert_equal %Q(<html><head><body>some more inputsome inputsome more input
some inputsome more inputsome inputsome more input
some inputsome more input<p>some input</p>some more input
some inputsome more input</body></html>), output

    assert_equal 7, tags.size
    include_tags = tags.select {|tag| tag[:name] == 'esi:include'}
    assert_equal 5, include_tags.size
    include_tags.each do|tag|
      assert_equal 'hello', tag[:attributes]['src']
    end

    inline_tags = tags.select {|tag| tag[:name] == 'esi:inline'}
    assert_equal 1, inline_tags.size
    comment_tags = tags.select {|tag| tag[:name] == 'esi:comment'}
    assert_equal 1, comment_tags.size

  end

  def test_block_parser_basics
    output = ""
    parser = ESIParser.new output
    tags = []
    parser.start_tag do|tag_name, attrs|
      tags << {:name => tag_name, :attributes => attrs}
    end
    parser.process "<html><head><body><esi:include timeout='1' max-age='600+600' src=\"hello\"/>some more input"
    parser.process "some input<esi:try><esi:attempt>some more input</esi:attempt><esi:except>\nsome input</esi:except></esi:try>some more input"
    parser.process "some input<esi:inline src='hello'/>some more input\nsome input<esi:comment text='hello'/>some more input"
    parser.process "<p>some input</p><esi:include src='hello'/>some more input\nsome input<esi:include src='hello'/>some more input"
    parser.process "</body></html>"
    parser.finish
    assert_equal %Q(<html><head><body>some more inputsome inputsome more input
some inputsome more inputsome inputsome more input
some inputsome more input<p>some input</p>some more input
some inputsome more input</body></html>), output
  end

  def test_empty_parse
    output = ""
    parser = ESIParser.new output
    tags = []
    parser.start_tag do|tag_name, attrs|
      tags << {:name => tag_name, :attributes => attrs}
    end
    assert_nothing_raised do
      parser.process ""
    end
    parser.finish
  end

  # it's a strange case but, we should still recongize the tag so that it can be pruned from the input stream
  def test_parser_accepts_empty_tag
    output = ""
    parser = ESIParser.new output
    tags = []
    parser.start_tag do|tag_name, attrs|
      tags << {:name => tag_name, :attributes => attrs}
    end
    parser.process "<p>some input</p><esi:include/>some more input\nsome input<esi:include src='hello'/>some more input"
    parser.finish
    assert_equal %Q(<p>some input</p>some more input
some inputsome more input), output

    assert_equal 2, tags.size
    assert_equal 'hello', tags.last[:attributes]['src']
  end

  def test_can_parse_in_chunks

    output = ""
    parser = ESIParser.new output
    tags = []
    parser.start_tag do|tag_name, attrs|
      tags << {:name => tag_name, :attributes => attrs}
    end
    parser.process "some input<esi:in"
    parser.process "line src='hel"
    parser.process "lo'"
    parser.process "/>some more input\nsome input<esi:comment text='hello'/>some more input"
    parser.finish

    assert_equal "some inputsome more input\nsome inputsome more input", output

    assert_equal 2, tags.size
    assert_equal 'hello', tags.first[:attributes]['src']

    output = ""
    parser = ESIParser.new output
    tags = []
    parser.start_tag do|tag_name, attrs|
      tags << {:name => tag_name, :attributes => attrs}
    end
    parser.process "some input<"
    parser.process "e"
    parser.process "s"
    parser.process "i"
    parser.process ":"
    parser.process "i"
    parser.process "n"
    parser.process "lin"
    parser.process "e"
    parser.process " "
    parser.process "s"
    parser.process "rc"
    parser.process "="
    parser.process "'hel"
    parser.process "lo'"
    parser.process "/"
    parser.process ">some more input\nsome input"
    parser.process "<esi:comment text="
    parser.process "'hello'/>some more input"
    parser.finish

    assert_equal "some inputsome more input\nsome inputsome more input", output

    assert_equal 2, tags.size
    assert_equal 1, tags.select{|tag| tag[:name] == 'esi:inline'}.size, "Failed to parse esi:inline"
    assert_equal 1, tags.select{|tag| tag[:name] == 'esi:comment'}.size, "Failed to parse esi:comment"
    assert_equal 'hello', tags.select{|tag| tag[:name] == 'esi:inline'}.first[:attributes]['src'], "Failed to parse esi:inline attributes"

  end

  def assert_totals_from_parse(parser)

    start_trys = 0
    start_attempts = 0
    start_includes = 0
    start_excepts = 0
    start_invalidates = 0
    parser.start_tag do|tag_name,attrs|
      #puts "start: #{tag_name.inspect}#{attrs.inspect}#{trys}"
      case tag_name
      when "esi:try" then start_trys += 1
      when "esi:attempt" then start_attempts += 1
      when "esi:include" then start_includes += 1
      when "esi:except" then start_excepts += 1
      when "esi:invalidate" then start_invalidates += 1
      else
        puts "Unverified start: #{tag_name.inspect}#{attrs.inspect}"
      end
    end

    end_trys = 0
    end_attempts = 0
    end_includes = 0
    end_excepts = 0
    end_invalidates = 0
    parser.end_tag do|tag_name|
      #puts "end: #{tag_name.inspect}"
      case tag_name
      when "esi:try" then end_trys += 1
      when "esi:attempt" then end_attempts += 1
      when "esi:include" then end_includes += 1
      when "esi:except" then end_excepts += 1
      when "esi:invalidate" then end_invalidates += 1
      else
        puts "Unverified start: #{tag_name.inspect}#{attrs.inspect}"
      end
    end

    yield 

    assert_equal 2, start_trys, "More or less esi:try tags detected #{start_trys.inspect}"
    assert_equal 2, start_attempts, "More or less esi:attempt tags detected #{start_attempts.inspect}"
    assert_equal 6, start_includes, "More or less esi:include tags detected #{start_includes.inspect}"
    assert_equal 2, start_excepts, "More or less esi:except tags detected #{start_excepts.inspect}"
    assert_equal 1, start_invalidates, "More or less esi:invalidate tags detected #{start_invalidates.inspect}"
    assert_equal 2, end_trys, "More or less esi:try tags detected #{end_trys.inspect}"
    assert_equal 2, end_attempts, "More or less esi:attempt tags detected #{end_attempts.inspect}"
    assert_equal 6, end_includes, "More or less esi:include tags detected #{end_includes.inspect}"
    assert_equal 2, end_excepts, "More or less esi:except tags detected #{end_excepts.inspect}"
    assert_equal 1, end_invalidates, "More or less esi:invalidate tags detected #{end_invalidates.inspect}"
  end

  def test_sample
    sample =  File.join(File.dirname(__FILE__),'esi-sample.html')
    output = ""
    parser = ESIParser.new output
    assert_totals_from_parse( parser ) do
      parser.process File.read(sample)
      parser.finish
    end
    assert_no_match /<esi:/, output

    #8.times do|i|
      i = 4
      File.open(sample,'r') do|input|
        output = ""
        parser = ESIParser.new output
        assert_totals_from_parse( parser ) do
          while( (buf=input.read((i+1)*2)) and !input.eof? and buf.size != 0 )
            parser.process buf
          end
          parser.process buf
          parser.finish
        end
        assert_no_match /<esi:/, output
      end
#    end

  end

end

=begin
TODO:
  * move code into a branch
  * merge into handler.rb, replacing stream_parser
=end
